Predict on unseen data

packages

pacman::p_load(dplyr, ggplot2, googlesheets, openxlsx, stringr, rvest, dplyr, ggplot2, h2o, caret, text2vec)

Installing package into 㤼㸱C:/Users/Workplace/Documents/R/win-library/3.4㤼㸲
(as 㤼㸱lib㤼㸲 is unspecified)
also installing the dependencies 㤼㸱lambda.r㤼㸲, 㤼㸱futile.options㤼㸲, 㤼㸱RcppParallel㤼㸲, 㤼㸱data.table㤼㸲, 㤼㸱irlba㤼㸲, 㤼㸱futile.logger㤼㸲, 㤼㸱mlapi㤼㸲, 㤼㸱sparsepp㤼㸲

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/lambda.r_1.2.zip'
Content type 'application/zip' length 93201 bytes (91 KB)
downloaded 91 KB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/futile.options_1.0.0.zip'
Content type 'application/zip' length 17090 bytes (16 KB)
downloaded 16 KB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/RcppParallel_4.3.20.zip'
Content type 'application/zip' length 3378535 bytes (3.2 MB)
downloaded 3.2 MB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/data.table_1.10.4-3.zip'
Content type 'application/zip' length 1577200 bytes (1.5 MB)
downloaded 1.5 MB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/irlba_2.3.2.zip'
Content type 'application/zip' length 279956 bytes (273 KB)
downloaded 273 KB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/futile.logger_1.4.3.zip'
Content type 'application/zip' length 95441 bytes (93 KB)
downloaded 93 KB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/mlapi_0.1.0.zip'
Content type 'application/zip' length 72925 bytes (71 KB)
downloaded 71 KB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/sparsepp_0.2.0.zip'
Content type 'application/zip' length 144909 bytes (141 KB)
downloaded 141 KB

trying URL 'https://cran.rstudio.com/bin/windows/contrib/3.4/text2vec_0.5.1.zip'
Content type 'application/zip' length 5140413 bytes (4.9 MB)
downloaded 4.9 MB

package ‘lambda.r’ successfully unpacked and MD5 sums checked
package ‘futile.options’ successfully unpacked and MD5 sums checked
package ‘RcppParallel’ successfully unpacked and MD5 sums checked
package ‘data.table’ successfully unpacked and MD5 sums checked
package ‘irlba’ successfully unpacked and MD5 sums checked
package ‘futile.logger’ successfully unpacked and MD5 sums checked
package ‘mlapi’ successfully unpacked and MD5 sums checked
package ‘sparsepp’ successfully unpacked and MD5 sums checked
package ‘text2vec’ successfully unpacked and MD5 sums checked

The downloaded binary packages are in
    C:\Users\Workplace\AppData\Local\Temp\RtmpGsflLY\downloaded_packages

text2vec installed
package 㤼㸱text2vec㤼㸲 was built under R version 3.4.3

load data

df <- get(load("sub_sample2.Rdata"))

clean

clean_social_media <- function(x){
  x %>%
    str_replace_all("\n", " ") %>%
    str_to_lower() %>%
    ### Twitter specific
    str_replace_all("https?[:]//[[:graph:]]+", "URL") %>%
    str_replace_all("@(\\w+)", " HNDL") %>%
    str_replace_all("#(\\w+)", " HASH") %>%
    ### ALtright replacements
    str_replace_all("\\(+.?", "JEW ") %>%
    str_replace_all("\\)+", " ") %>%
    ### smilies
    str_replace_all(":-\\)|:\\)|\\(:|\\(-:", " EMO_SMILEY ") %>%
    str_replace_all(":-D|:D|X-D|XD|xD", " EMO_LAUGH ") %>%
    str_replace_all("<3|:\\*", "EMO_LOVE") %>%
    str_replace_all(";-\\)|;\\)|;-D|;D|\\(;|\\(-;", "EMO_WINK") %>%
    str_replace_all(":-\\(|:\\(|\\):|\\)-:", "EMO_FROWN") %>%
    str_replace_all(':,\\(|:"\\(|:\\(\\(', "EMO_CRY") %>%
    ### General
    str_replace_all("\\.|\\:|\\;", " PUNC_DOT ") %>%
    str_replace_all("\\!", " PUNC_EXCL ") %>%
    str_replace_all("\\?", " PUNC_QUES ") %>%
    str_replace_all("\\.\\.\\.", " PUNC_DOTS ") %>%
    str_trim() 
}
df <- df %>%
  mutate(ctext = clean_social_media(text))

package 㤼㸱bindrcpp㤼㸲 was built under R version 3.4.3

vectorize

load("vectorizer.Rdata")
### test
pred_it <- itoken(
  df$ctext, 
  ids = df$id,
  progressbar = F
)
pred_dtm <- create_dtm(pred_it, vectorizer)

predict

df_pred %>% split(., .[,"sp"])

$`1`

$`2`

$`3`

$`4`

$`5`

$`6`

$`7`

$`8`

$`9`

$`10`

$`11`

$`12`

$`13`

$`14`

$`15`

$`16`

$`17`

$`18`

$`19`

$`20`

NA